# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from pyquery import PyQuery as pq
import re
from scrapy.http import Request as sreq
from scrapy.http import FormRequest as fr
from girl.items.dmm import *
from scrapy.shell import inspect_response
import requests as req
import time
import datetime
import platform
import redis
from scrapy_redis.spiders import RedisCrawlSpider
def process_value(value):
    m = re.search(".*/detail/=/cid=.+/",value)
    if m:
        return m.group()
class DmmSpider(RedisCrawlSpider):
    name = "dmm"
    allowed_domains = ["dmm.co.jp"]
    url = "http://www.dmm.co.jp/digital/%s/-/list/=/sort=%s/"
    start_urls = [url % (x,y) for x in ('videoa','anime','videoc','nikkatsu') for y in
    ('date','ranking','saleranking_asc','review_rank','bookmark_desc','price_asc')]
    start_urls += ["http://www.dmm.co.jp/digital/videoc/-/list/=/device=android/limit=120/n1=DgRJTglEBQ4GpteF45LR/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4GpoD6,YyI,qs_/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4GwP6dl5O1ktnA0ZPD/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4Go,n42fG9iM8_/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=tv/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=ios/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=android/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=vita/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=tv/limit=120/n1=DgRJTglEBQ4GwP6dl5O1ktnA0ZPD/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/device=vr/sort=ranking/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=1014/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=2001/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=1034/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=1039/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=1018/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=5001/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=5005/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=4015/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=4001/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/article=keyword/id=6793/limit=120/sort=saleranking_asc/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4GwP6dl5O1ktnA0ZPD/n2=DgRJTglEBQ4Grc*GnIuR/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4GwP6dl5O1ktnA0ZPD/n2=DgRJTglEBQ4Grc*GnIuR/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4GwP6dl5O1ktnA0ZPD/n2=DgRJTglEBQ4GpPaH,g__/sort=date/"]
    start_urls += ["http://www.dmm.co.jp/digital/videoa/-/list/=/limit=120/n1=DgRJTglEBQ4G2P6FxA__/sort=date/"]

    pool = redis.ConnectionPool(host='localhost', port=6379)
    conn = redis.Redis(connection_pool=pool)
    conn.sadd("dmm:start_urls",*start_urls)
    redis_key = "dmm:start_urls"

    rules = (
        Rule(LinkExtractor(
        allow=('.*digital/(videoa|anime|videoc|nikkatsu)/.*'),
        deny=('.*digital/-/basket.*'),
        restrict_xpaths="//div[@class='list-boxcaptside list-boxpagenation']//a")),
        # Rule(LinkExtractor(
        # allow=('.*digital/(videoa|anime|videoc|nikkatsu)/.*'),
        # deny=('.*digital/-/basket.*'),
        # restrict_xpaths="//div[@class='side-menu'][0]//a")),
        Rule(LinkExtractor(
        allow=('.*digital/(videoa|anime|videoc|nikkatsu)/-/detail/=/cid=.+/'),
        process_value=process_value),
        callback='video')
    )
    # deny=('.*digital/-/basket.*|.*page=([1-3][0-3][0-8]|[1-9][0-9]|[1-9]|)/?'),
    tmp = {'girl.newdatapipelines.NewDataPipeline': 300}
    custom_settings = {
        'ITEM_PIPELINES':tmp,
        'RETRY_ENABLED' : False,
        # "DOWNLOAD_TIMEOUT": 380,
        "CONCURRENT_REQUESTS":200,
        # "HTTPCACHE_IGNORE_MISSING":True
        "HTTPCACHE_EXPIRATION_SECS":60 * 60 * 24 * 300
    }

    if platform.system() != 'Darwin':
        time = datetime.datetime.now().strftime('%m-%d-%H-%M')
        custom_settings["LOG_FILE"] = "/mnt/scrapy/crawler/dmm-%s.log" % time

    def __init__(self, *a, **k):
        # if k.get("type"):

        super(DmmSpider, self).__init__(*a, **k)


    def video(self,response):
        item = girlItem()
        x = response.xpath
        tmp = x("//table[@class='mg-b20']//tr//text()").extract()
        key = ""
        item["title"] = x("//h1[@id='title']/text()").extract_first()
        item["name"] = ""
        item["bwh"] = ""
        reg = re.compile(r"^\s+$")
        for da in tmp:
            if reg.search(da):
                continue
            if key:
                item[key] = da
                key = ""
            if da.find("対応デバイス") >= 0:
                key = "apply"
            # elif da == "配信開始日":
                # key = "title"
            elif da.find("配信開始日") >= 0:
                key = "release"
            elif da.find("収録時間") >= 0:
                key = "time"
            elif da.find("監督") >= 0:
                key = "director"
            elif da.find("名前") >= 0:
                key = "name"
            elif da.find("サイズ") >= 0:
                key = "bwh"
            elif da.find("品番") >= 0:
                key = "designation"
            elif da.find("レーベル") >= 0:
                key = "tag"
            elif da.find("メーカー") >= 0:
                key = "manufacturer"
            elif da.find("シリーズ") >= 0:
                key = "series"
        item["date"] = str(datetime.date.today())
        item["bigImg"] = x("//*[@id='sample-video']/a/@href").extract_first()
        # if not item["bigImg"]:
            # import ipdb;ipdb.set_trace()
        item["play"] = x("//*[@id='performer']/a/text()").extract()
        item["cate"] = x("//table[@class='mg-b20']//tr//a[contains(@href,'keyword')]/text()").extract()
        item["imgList"] = x("//div[@id='sample-image-block']//img/@src").extract()
        item["url"] = response.url
        item["thumb"] = x("//div[@id='sample-video']//img/@src").extract_first()
        tmp = x("//a[@class='d-btn']//@onclick").extract()
        if tmp:
            item["video"] = re.search(r"\('(.*)'\)",tmp[0]).group(1)
        yield item

